{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[WARNING] ME(12002:140052698522816,MainProcess):2024-05-14-19:50:24.413.323 [mindspore/run_check/_check_version.py:102] MindSpore version 2.2.13 and cuda version 11.8.89 does not match, CUDA version [['10.1', '11.1', '11.6']] are supported by MindSpore officially. Please refer to the installation guide for version matching information: https://www.mindspore.cn/install.\n",
      "/hpc2hdd/home/ypeng455/.conda/envs/py39/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache /tmp/jieba.cache\n",
      "Loading model cost 0.702 seconds.\n",
      "Prefix dict has been built successfully.\n",
      "Loading checkpoint shards: 100%|██████████| 8/8 [01:06<00:00,  8.26s/it]\n",
      "The following parameters in checkpoint files are not loaded:\n",
      "['model.layers.0.self_attn.rotary_emb.inv_freq', 'model.layers.1.self_attn.rotary_emb.inv_freq', 'model.layers.2.self_attn.rotary_emb.inv_freq', 'model.layers.3.self_attn.rotary_emb.inv_freq', 'model.layers.4.self_attn.rotary_emb.inv_freq', 'model.layers.5.self_attn.rotary_emb.inv_freq', 'model.layers.10.self_attn.rotary_emb.inv_freq', 'model.layers.11.self_attn.rotary_emb.inv_freq', 'model.layers.6.self_attn.rotary_emb.inv_freq', 'model.layers.7.self_attn.rotary_emb.inv_freq', 'model.layers.8.self_attn.rotary_emb.inv_freq', 'model.layers.9.self_attn.rotary_emb.inv_freq', 'model.layers.12.self_attn.rotary_emb.inv_freq', 'model.layers.13.self_attn.rotary_emb.inv_freq', 'model.layers.14.self_attn.rotary_emb.inv_freq', 'model.layers.15.self_attn.rotary_emb.inv_freq', 'model.layers.16.self_attn.rotary_emb.inv_freq', 'model.layers.17.self_attn.rotary_emb.inv_freq', 'model.layers.18.self_attn.rotary_emb.inv_freq', 'model.layers.19.self_attn.rotary_emb.inv_freq', 'model.layers.20.self_attn.rotary_emb.inv_freq', 'model.layers.21.self_attn.rotary_emb.inv_freq', 'model.layers.22.self_attn.rotary_emb.inv_freq', 'model.layers.23.self_attn.rotary_emb.inv_freq', 'model.layers.24.self_attn.rotary_emb.inv_freq', 'model.layers.25.self_attn.rotary_emb.inv_freq', 'model.layers.26.self_attn.rotary_emb.inv_freq', 'model.layers.27.self_attn.rotary_emb.inv_freq', 'model.layers.28.self_attn.rotary_emb.inv_freq', 'model.layers.29.self_attn.rotary_emb.inv_freq', 'model.layers.30.self_attn.rotary_emb.inv_freq', 'model.layers.31.self_attn.rotary_emb.inv_freq']\n",
      "/hpc2hdd/home/ypeng455/mindnlpV3/mindnlp/mindnlp/transformers/generation/utils.py:1402: UserWarning: You have modified the pretrained model configuration to control generation. This is a deprecated strategy to control generation. Please use and modify the model generation configuration (see https://hf-mirror.com/docs/transformers/generation_strategies#default-text-generation-configuration )\n",
      "  warnings.warn(\n",
      "[WARNING] KERNEL(12002,7f5e1f7fe700,python):2024-05-14-19:51:46.841.106 [mindspore/ccsrc/kernel/kernel.h:376] CheckShapeNull] For 'Equal', the shape of input_0 cannot contain zero, but got [const vector]{1, 0}\n",
      "[WARNING] KERNEL(12002,7f608f5864c0,python):2024-05-14-19:51:46.847.655 [mindspore/ccsrc/kernel/kernel.h:376] CheckShapeNull] For 'BitwiseAnd', the shape of input_0 cannot contain zero, but got [const vector]{1, 0}\n",
      "[WARNING] KERNEL(12002,7f608f5864c0,python):2024-05-14-19:51:46.848.234 [mindspore/ccsrc/kernel/kernel.h:376] CheckShapeNull] For 'Cast', the shape of input cannot contain zero, but got [const vector]{1, 0}\n",
      "[WARNING] KERNEL(12002,7f5f09687700,python):2024-05-14-19:51:46.997.504 [mindspore/ccsrc/kernel/kernel.h:376] CheckShapeNull] For 'Mul', the shape of input_0 cannot contain zero, but got [const vector]{0, 11008}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The image is a flowchart that illustrates the process of how a visual language model and a visual expert built on a language model work. On the left, the visual language model takes an image and a textual description, processes them through an MLP adapter, and then through a VT encoder and a word embedding layer to produce a paired text. On the right, the visual expert model takes an image and a textual description, processes them through a multi-head attention mechanism with a QKV matrix, and then through a layer norm and text features to produce a text.</s>\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "以下是推理结果，有几个warning，但推理结果和torch一致\n",
    "\"\"\"\n",
    "from mindnlp.transformers import CogVLMConfig,MLP,RMSNorm,PretrainedConfig,LlamaTokenizer,AutoModelForCausalLM\n",
    "import mindspore\n",
    "mindspore.set_seed(42)\n",
    "ms_net = AutoModelForCausalLM.from_pretrained(\n",
    "    'THUDM/cogvlm-chat-hf'\n",
    ")\n",
    "from PIL import Image\n",
    "tokenizer = LlamaTokenizer.from_pretrained('lmsys/vicuna-7b-v1.5')\n",
    "query = 'describe this picture'\n",
    "image = Image.open('CogVLM.png').convert('RGB')\n",
    "ms_inputs = ms_net.build_conversation_input_ids(tokenizer, query=query, history=[],images=[image])  # chat mode\n",
    "\n",
    "ms_inputs = {\n",
    "    'input_ids': ms_inputs['input_ids'].unsqueeze(0),\n",
    "    'token_type_ids': ms_inputs['token_type_ids'].unsqueeze(0),\n",
    "    'attention_mask': ms_inputs['attention_mask'].unsqueeze(0),\n",
    "    'images': [[mindspore.Tensor(ms_inputs['images'][0],dtype=mindspore.float16)]],\n",
    "}\n",
    "\n",
    "gen_kwargs = {\"max_length\": 2048, \"do_sample\": False}\n",
    "ms_out = ms_net.generate(**ms_inputs,**gen_kwargs)\n",
    "outputs = ms_out[:, ms_inputs['input_ids'].shape[1]:]\n",
    "print(tokenizer.decode(outputs[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "你好! 很感谢您咨询. 请问您具备哪些��</s>\n"
     ]
    }
   ],
   "source": [
    "query = '你好'\n",
    "text_only_template = f\"A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. USER: {query} ASSISTANT:\" \n",
    "query = text_only_template\n",
    "image = Image.open('/hpc2hdd/home/ypeng455/mindnlp/mindnlp/transformers/models/cogvlm/CogVLM.png').convert('RGB')\n",
    "ms_inputs = ms_net.build_conversation_input_ids(tokenizer, query=query, history=[]) # chat mode\n",
    "\n",
    "ms_inputs = {\n",
    "    'input_ids': ms_inputs['input_ids'].unsqueeze(0),\n",
    "    'token_type_ids': ms_inputs['token_type_ids'].unsqueeze(0),\n",
    "    'attention_mask': ms_inputs['attention_mask'].unsqueeze(0),\n",
    "}\n",
    "\n",
    "gen_kwargs = {\"max_length\": 2048, \"do_sample\": False}\n",
    "ms_out = ms_net.generate(**ms_inputs,**gen_kwargs)\n",
    "outputs = ms_out[:, ms_inputs['input_ids'].shape[1]:]\n",
    "print(tokenizer.decode(outputs[0]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.19 ('py39': conda)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.19"
  },
  "vscode": {
   "interpreter": {
    "hash": "f688b49bf7bbd372001c59148eb4c8aaba45f80791d96530eef356c517b27051"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
